{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2b9efac3",
   "metadata": {},
   "source": [
    "### 模型微调代码优化："
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfaa4395",
   "metadata": {},
   "source": [
    "优化内容：使用Trainer+TrainingArgument优化训练流程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "07813548",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7252375691ab46168011e93cde7ffd16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/6988 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8439dcbed2c34980a1e3e2fc601c9f7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/777 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 第一步到第五步\n",
    "from transformers import AutoTokenizer,AutoModelForSequenceClassification,Trainer,TrainingArguments\n",
    "from datasets import load_dataset\n",
    "dataset=load_dataset(\"csv\",data_files=\"tst_fold\\ChnSentiCorp_htl_all copy.csv\",split=\"train\")\n",
    "dataset=dataset.filter(lambda x: x[\"review\"] is not None)\n",
    "datasets=dataset.train_test_split(test_size=0.1)\n",
    "import torch\n",
    "tokenizer=AutoTokenizer.from_pretrained(r\"E:\\HuggFace_model\\rbt3\")\n",
    "def process_func(examples):\n",
    "    tokenized_examples=tokenizer(examples[\"review\"],max_length=128,truncation=True)\n",
    "    tokenized_examples[\"labels\"]=examples[\"label\"]\n",
    "    return tokenized_examples\n",
    "tokenized_datasets=datasets.map(process_func,batched=True,remove_columns=datasets[\"train\"].column_names)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6454ce65",
   "metadata": {},
   "source": [
    "对比以往删掉了dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "da9a907a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at E:\\HuggFace_model\\rbt3 were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at E:\\HuggFace_model\\rbt3 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"创建模型\"\"\"\n",
    "model=AutoModelForSequenceClassification.from_pretrained(r\"E:\\HuggFace_model\\rbt3\")#模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08ebfb29",
   "metadata": {},
   "source": [
    "**创建评估函数**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "21e1e1cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "acc_metrics=evaluate.load(r\"evaluate_main\\metrics\\accuracy\")\n",
    "f1_metrics=evaluate.load(r\"evaluate_main\\metrics\\f1\")\n",
    "def eval_metric(eval_predict):\n",
    "    predictions,labels=eval_predict\n",
    "    predictions=predictions.argmax(axis=-1)\n",
    "    acc=acc_metrics.compute(predictions=predictions,references=labels)\n",
    "    f1=f1_metrics.compute(predictions=predictions,references=labels)\n",
    "    acc.update(f1)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81c06121",
   "metadata": {},
   "source": [
    "**创建TrainingArguments**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "65e0da3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TrainingArguments(\n",
       "_n_gpu=1,\n",
       "adafactor=False,\n",
       "adam_beta1=0.9,\n",
       "adam_beta2=0.999,\n",
       "adam_epsilon=1e-08,\n",
       "auto_find_batch_size=False,\n",
       "bf16=False,\n",
       "bf16_full_eval=False,\n",
       "data_seed=None,\n",
       "dataloader_drop_last=False,\n",
       "dataloader_num_workers=0,\n",
       "dataloader_pin_memory=True,\n",
       "ddp_bucket_cap_mb=None,\n",
       "ddp_find_unused_parameters=None,\n",
       "ddp_timeout=1800,\n",
       "debug=[],\n",
       "deepspeed=None,\n",
       "disable_tqdm=False,\n",
       "do_eval=True,\n",
       "do_predict=False,\n",
       "do_train=False,\n",
       "eval_accumulation_steps=None,\n",
       "eval_delay=0,\n",
       "eval_steps=None,\n",
       "evaluation_strategy=epoch,\n",
       "fp16=False,\n",
       "fp16_backend=auto,\n",
       "fp16_full_eval=False,\n",
       "fp16_opt_level=O1,\n",
       "fsdp=[],\n",
       "fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\n",
       "fsdp_min_num_params=0,\n",
       "fsdp_transformer_layer_cls_to_wrap=None,\n",
       "full_determinism=False,\n",
       "gradient_accumulation_steps=1,\n",
       "gradient_checkpointing=False,\n",
       "greater_is_better=True,\n",
       "group_by_length=False,\n",
       "half_precision_backend=auto,\n",
       "hub_model_id=None,\n",
       "hub_private_repo=False,\n",
       "hub_strategy=every_save,\n",
       "hub_token=<HUB_TOKEN>,\n",
       "ignore_data_skip=False,\n",
       "include_inputs_for_metrics=False,\n",
       "jit_mode_eval=False,\n",
       "label_names=None,\n",
       "label_smoothing_factor=0.0,\n",
       "learning_rate=5e-05,\n",
       "length_column_name=length,\n",
       "load_best_model_at_end=True,\n",
       "local_rank=-1,\n",
       "log_level=passive,\n",
       "log_level_replica=warning,\n",
       "log_on_each_node=True,\n",
       "logging_dir=./checkpoints_Trainer\\runs\\May12_18-29-41_DESKTOP-22GG4OF,\n",
       "logging_first_step=False,\n",
       "logging_nan_inf_filter=True,\n",
       "logging_steps=10,\n",
       "logging_strategy=steps,\n",
       "lr_scheduler_type=linear,\n",
       "max_grad_norm=1.0,\n",
       "max_steps=-1,\n",
       "metric_for_best_model=f1,\n",
       "mp_parameters=,\n",
       "no_cuda=False,\n",
       "num_train_epochs=5,\n",
       "optim=adamw_hf,\n",
       "optim_args=None,\n",
       "output_dir=./checkpoints_Trainer,\n",
       "overwrite_output_dir=False,\n",
       "past_index=-1,\n",
       "per_device_eval_batch_size=64,\n",
       "per_device_train_batch_size=32,\n",
       "prediction_loss_only=False,\n",
       "push_to_hub=False,\n",
       "push_to_hub_model_id=None,\n",
       "push_to_hub_organization=None,\n",
       "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
       "ray_scope=last,\n",
       "remove_unused_columns=True,\n",
       "report_to=['tensorboard'],\n",
       "resume_from_checkpoint=None,\n",
       "run_name=./checkpoints_Trainer,\n",
       "save_on_each_node=False,\n",
       "save_steps=500,\n",
       "save_strategy=epoch,\n",
       "save_total_limit=5,\n",
       "seed=42,\n",
       "sharded_ddp=[],\n",
       "skip_memory_metrics=True,\n",
       "tf32=None,\n",
       "torch_compile=False,\n",
       "torch_compile_backend=None,\n",
       "torch_compile_mode=None,\n",
       "torchdynamo=None,\n",
       "tpu_metrics_debug=False,\n",
       "tpu_num_cores=None,\n",
       "use_ipex=False,\n",
       "use_legacy_prediction_loop=False,\n",
       "use_mps_device=False,\n",
       "warmup_ratio=0.0,\n",
       "warmup_steps=0,\n",
       "weight_decay=0.01,\n",
       "xpu_backend=None,\n",
       ")"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_args=TrainingArguments(output_dir=\"./checkpoints_Trainer\",\n",
    "                             per_device_train_batch_size=32,#训练集 dataloader每个批次大小\n",
    "                             per_device_eval_batch_size=64,#测试集 dataloader 每个批次大小\n",
    "                             num_train_epochs=5, #总轮数\n",
    "                             logging_steps=10, # 每十步进行一次打印\n",
    "                             #epoch:每一轮做一次评估\n",
    "                             #steps，需要搭配eval_steps=xxx使用:没xxx步评估一次\n",
    "                             evaluation_strategy=\"epoch\",\n",
    "                             #保存策略\n",
    "                             save_strategy=\"epoch\",#每一轮保存一下模型\n",
    "                             save_total_limit=5,#限制最多保存5个模型\n",
    "                             weight_decay=0.01,#梯度衰减\n",
    "                             metric_for_best_model=\"f1\",#设置评估指标\n",
    "                             load_best_model_at_end=True#最后默认加载最好的 模型 \n",
    "                            #  optim= 设置优化器\n",
    "                             )\n",
    "train_args#查看参数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04a6fb11",
   "metadata": {},
   "source": [
    "**创建Trainer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9556ccdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorWithPadding\n",
    "\"\"\"传入模型，训练参数，数据集\"\"\"\n",
    "trainer=Trainer(model=model,args=train_args,train_dataset=tokenized_datasets[\"train\"],\n",
    "                eval_dataset=tokenized_datasets[\"test\"],\n",
    "                data_collator=DataCollatorWithPadding(tokenizer=tokenizer),\n",
    "                compute_metrics=eval_metric \n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5eb699c",
   "metadata": {},
   "source": [
    "**模型训练**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "edeefbbd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "z:\\ANACONDA\\envs\\transformers\\lib\\site-packages\\transformers\\optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05236082eea341f78741763e70182803",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1095 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'learning_rate': 4.9543378995433794e-05, 'epoch': 0.05}\n",
      "{'loss': 0.0656, 'learning_rate': 4.908675799086758e-05, 'epoch': 0.09}\n",
      "{'loss': 0.0213, 'learning_rate': 4.863013698630137e-05, 'epoch': 0.14}\n",
      "{'loss': 0.1677, 'learning_rate': 4.8173515981735164e-05, 'epoch': 0.18}\n",
      "{'loss': 0.0015, 'learning_rate': 4.7716894977168955e-05, 'epoch': 0.23}\n",
      "{'loss': 0.0021, 'learning_rate': 4.726027397260274e-05, 'epoch': 0.27}\n",
      "{'loss': 0.0002, 'learning_rate': 4.680365296803653e-05, 'epoch': 0.32}\n",
      "{'loss': 0.0015, 'learning_rate': 4.6347031963470325e-05, 'epoch': 0.37}\n",
      "{'loss': 0.0129, 'learning_rate': 4.589041095890411e-05, 'epoch': 0.41}\n",
      "{'loss': 0.051, 'learning_rate': 4.54337899543379e-05, 'epoch': 0.46}\n",
      "{'loss': 0.0374, 'learning_rate': 4.4977168949771694e-05, 'epoch': 0.5}\n",
      "{'loss': 0.03, 'learning_rate': 4.452054794520548e-05, 'epoch': 0.55}\n",
      "{'loss': 0.0222, 'learning_rate': 4.406392694063927e-05, 'epoch': 0.59}\n",
      "{'loss': 0.0312, 'learning_rate': 4.360730593607306e-05, 'epoch': 0.64}\n",
      "{'loss': 0.0208, 'learning_rate': 4.3150684931506855e-05, 'epoch': 0.68}\n",
      "{'loss': 0.001, 'learning_rate': 4.269406392694064e-05, 'epoch': 0.73}\n",
      "{'loss': 0.0329, 'learning_rate': 4.223744292237443e-05, 'epoch': 0.78}\n",
      "{'loss': 0.0482, 'learning_rate': 4.1780821917808224e-05, 'epoch': 0.82}\n",
      "{'loss': 0.0303, 'learning_rate': 4.132420091324201e-05, 'epoch': 0.87}\n",
      "{'loss': 0.0663, 'learning_rate': 4.08675799086758e-05, 'epoch': 0.91}\n",
      "{'loss': 0.0413, 'learning_rate': 4.041095890410959e-05, 'epoch': 0.96}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e68f35f9bbc4f929d9c787aaf8a347f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6429142951965332, 'eval_accuracy': 0.9099099099099099, 'eval_f1': 0.9351851851851852, 'eval_runtime': 3.8263, 'eval_samples_per_second': 203.067, 'eval_steps_per_second': 3.398, 'epoch': 1.0}\n",
      "{'loss': 0.0307, 'learning_rate': 3.995433789954338e-05, 'epoch': 1.0}\n",
      "{'loss': 0.0391, 'learning_rate': 3.949771689497717e-05, 'epoch': 1.05}\n",
      "{'loss': 0.0697, 'learning_rate': 3.904109589041096e-05, 'epoch': 1.1}\n",
      "{'loss': 0.0766, 'learning_rate': 3.8584474885844754e-05, 'epoch': 1.14}\n",
      "{'loss': 0.019, 'learning_rate': 3.812785388127854e-05, 'epoch': 1.19}\n",
      "{'loss': 0.008, 'learning_rate': 3.767123287671233e-05, 'epoch': 1.23}\n",
      "{'loss': 0.0326, 'learning_rate': 3.7214611872146123e-05, 'epoch': 1.28}\n",
      "{'loss': 0.0213, 'learning_rate': 3.675799086757991e-05, 'epoch': 1.32}\n",
      "{'loss': 0.0726, 'learning_rate': 3.63013698630137e-05, 'epoch': 1.37}\n",
      "{'loss': 0.032, 'learning_rate': 3.584474885844749e-05, 'epoch': 1.42}\n",
      "{'loss': 0.0069, 'learning_rate': 3.538812785388128e-05, 'epoch': 1.46}\n",
      "{'loss': 0.0065, 'learning_rate': 3.493150684931507e-05, 'epoch': 1.51}\n",
      "{'loss': 0.0006, 'learning_rate': 3.447488584474886e-05, 'epoch': 1.55}\n",
      "{'loss': 0.0092, 'learning_rate': 3.4018264840182654e-05, 'epoch': 1.6}\n",
      "{'loss': 0.0051, 'learning_rate': 3.356164383561644e-05, 'epoch': 1.64}\n",
      "{'loss': 0.001, 'learning_rate': 3.310502283105023e-05, 'epoch': 1.69}\n",
      "{'loss': 0.0041, 'learning_rate': 3.264840182648402e-05, 'epoch': 1.74}\n",
      "{'loss': 0.0099, 'learning_rate': 3.219178082191781e-05, 'epoch': 1.78}\n",
      "{'loss': 0.0356, 'learning_rate': 3.17351598173516e-05, 'epoch': 1.83}\n",
      "{'loss': 0.0539, 'learning_rate': 3.127853881278539e-05, 'epoch': 1.87}\n",
      "{'loss': 0.0001, 'learning_rate': 3.082191780821918e-05, 'epoch': 1.92}\n",
      "{'loss': 0.0058, 'learning_rate': 3.036529680365297e-05, 'epoch': 1.96}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89bac06d13554406a4c5d6874f283a60",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7958603501319885, 'eval_accuracy': 0.8906048906048906, 'eval_f1': 0.9216589861751152, 'eval_runtime': 3.9158, 'eval_samples_per_second': 198.427, 'eval_steps_per_second': 3.32, 'epoch': 2.0}\n",
      "{'loss': 0.0014, 'learning_rate': 2.990867579908676e-05, 'epoch': 2.01}\n",
      "{'loss': 0.0133, 'learning_rate': 2.945205479452055e-05, 'epoch': 2.05}\n",
      "{'loss': 0.0009, 'learning_rate': 2.8995433789954342e-05, 'epoch': 2.1}\n",
      "{'loss': 0.0016, 'learning_rate': 2.853881278538813e-05, 'epoch': 2.15}\n",
      "{'loss': 0.0061, 'learning_rate': 2.808219178082192e-05, 'epoch': 2.19}\n",
      "{'loss': 0.0165, 'learning_rate': 2.762557077625571e-05, 'epoch': 2.24}\n",
      "{'loss': 0.0003, 'learning_rate': 2.71689497716895e-05, 'epoch': 2.28}\n",
      "{'loss': 0.0152, 'learning_rate': 2.671232876712329e-05, 'epoch': 2.33}\n",
      "{'loss': 0.0306, 'learning_rate': 2.625570776255708e-05, 'epoch': 2.37}\n",
      "{'loss': 0.0199, 'learning_rate': 2.579908675799087e-05, 'epoch': 2.42}\n",
      "{'loss': 0.0004, 'learning_rate': 2.534246575342466e-05, 'epoch': 2.47}\n",
      "{'loss': 0.0031, 'learning_rate': 2.4885844748858446e-05, 'epoch': 2.51}\n",
      "{'loss': 0.0273, 'learning_rate': 2.4429223744292238e-05, 'epoch': 2.56}\n",
      "{'loss': 0.0264, 'learning_rate': 2.3972602739726026e-05, 'epoch': 2.6}\n",
      "{'loss': 0.0461, 'learning_rate': 2.351598173515982e-05, 'epoch': 2.65}\n",
      "{'loss': 0.0033, 'learning_rate': 2.3059360730593607e-05, 'epoch': 2.69}\n",
      "{'loss': 0.0063, 'learning_rate': 2.2602739726027396e-05, 'epoch': 2.74}\n",
      "{'loss': 0.0018, 'learning_rate': 2.2146118721461187e-05, 'epoch': 2.79}\n",
      "{'loss': 0.0244, 'learning_rate': 2.1689497716894976e-05, 'epoch': 2.83}\n",
      "{'loss': 0.0002, 'learning_rate': 2.1232876712328768e-05, 'epoch': 2.88}\n",
      "{'loss': 0.0002, 'learning_rate': 2.0776255707762557e-05, 'epoch': 2.92}\n",
      "{'loss': 0.01, 'learning_rate': 2.0319634703196345e-05, 'epoch': 2.97}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "412b032cc678434c884b68d79ae9fb80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.846077024936676, 'eval_accuracy': 0.8957528957528957, 'eval_f1': 0.925756186984418, 'eval_runtime': 3.8963, 'eval_samples_per_second': 199.423, 'eval_steps_per_second': 3.337, 'epoch': 3.0}\n",
      "{'loss': 0.0002, 'learning_rate': 1.9863013698630137e-05, 'epoch': 3.01}\n",
      "{'loss': 0.0023, 'learning_rate': 1.9406392694063926e-05, 'epoch': 3.06}\n",
      "{'loss': 0.017, 'learning_rate': 1.8949771689497718e-05, 'epoch': 3.11}\n",
      "{'loss': 0.0005, 'learning_rate': 1.8493150684931506e-05, 'epoch': 3.15}\n",
      "{'loss': 0.0041, 'learning_rate': 1.80365296803653e-05, 'epoch': 3.2}\n",
      "{'loss': 0.0009, 'learning_rate': 1.7579908675799087e-05, 'epoch': 3.24}\n",
      "{'loss': 0.0018, 'learning_rate': 1.7123287671232875e-05, 'epoch': 3.29}\n",
      "{'loss': 0.0005, 'learning_rate': 1.6666666666666667e-05, 'epoch': 3.33}\n",
      "{'loss': 0.0182, 'learning_rate': 1.6210045662100456e-05, 'epoch': 3.38}\n",
      "{'loss': 0.0015, 'learning_rate': 1.5753424657534248e-05, 'epoch': 3.42}\n",
      "{'loss': 0.0, 'learning_rate': 1.5296803652968037e-05, 'epoch': 3.47}\n",
      "{'loss': 0.001, 'learning_rate': 1.4840182648401827e-05, 'epoch': 3.52}\n",
      "{'loss': 0.002, 'learning_rate': 1.4383561643835617e-05, 'epoch': 3.56}\n",
      "{'loss': 0.0086, 'learning_rate': 1.3926940639269406e-05, 'epoch': 3.61}\n",
      "{'loss': 0.0002, 'learning_rate': 1.3470319634703196e-05, 'epoch': 3.65}\n",
      "{'loss': 0.0097, 'learning_rate': 1.3013698630136986e-05, 'epoch': 3.7}\n",
      "{'loss': 0.0015, 'learning_rate': 1.2557077625570777e-05, 'epoch': 3.74}\n",
      "{'loss': 0.0, 'learning_rate': 1.2100456621004567e-05, 'epoch': 3.79}\n",
      "{'loss': 0.0049, 'learning_rate': 1.1643835616438355e-05, 'epoch': 3.84}\n",
      "{'loss': 0.0063, 'learning_rate': 1.1187214611872146e-05, 'epoch': 3.88}\n",
      "{'loss': 0.0007, 'learning_rate': 1.0730593607305936e-05, 'epoch': 3.93}\n",
      "{'loss': 0.0006, 'learning_rate': 1.0273972602739726e-05, 'epoch': 3.97}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db16e53b9287457b914d05860019f8b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8466149568557739, 'eval_accuracy': 0.8944658944658944, 'eval_f1': 0.9239332096474954, 'eval_runtime': 3.8847, 'eval_samples_per_second': 200.014, 'eval_steps_per_second': 3.346, 'epoch': 4.0}\n",
      "{'loss': 0.0001, 'learning_rate': 9.817351598173517e-06, 'epoch': 4.02}\n",
      "{'loss': 0.0019, 'learning_rate': 9.360730593607305e-06, 'epoch': 4.06}\n",
      "{'loss': 0.0002, 'learning_rate': 8.904109589041095e-06, 'epoch': 4.11}\n",
      "{'loss': 0.0, 'learning_rate': 8.447488584474886e-06, 'epoch': 4.16}\n",
      "{'loss': 0.0001, 'learning_rate': 7.990867579908676e-06, 'epoch': 4.2}\n",
      "{'loss': 0.0, 'learning_rate': 7.5342465753424655e-06, 'epoch': 4.25}\n",
      "{'loss': 0.0003, 'learning_rate': 7.077625570776256e-06, 'epoch': 4.29}\n",
      "{'loss': 0.0078, 'learning_rate': 6.621004566210046e-06, 'epoch': 4.34}\n",
      "{'loss': 0.0072, 'learning_rate': 6.1643835616438354e-06, 'epoch': 4.38}\n",
      "{'loss': 0.0, 'learning_rate': 5.707762557077626e-06, 'epoch': 4.43}\n",
      "{'loss': 0.0, 'learning_rate': 5.251141552511415e-06, 'epoch': 4.47}\n",
      "{'loss': 0.0023, 'learning_rate': 4.7945205479452054e-06, 'epoch': 4.52}\n",
      "{'loss': 0.0001, 'learning_rate': 4.337899543378996e-06, 'epoch': 4.57}\n",
      "{'loss': 0.0039, 'learning_rate': 3.881278538812785e-06, 'epoch': 4.61}\n",
      "{'loss': 0.0074, 'learning_rate': 3.4246575342465754e-06, 'epoch': 4.66}\n",
      "{'loss': 0.0123, 'learning_rate': 2.9680365296803653e-06, 'epoch': 4.7}\n",
      "{'loss': 0.0384, 'learning_rate': 2.511415525114155e-06, 'epoch': 4.75}\n",
      "{'loss': 0.0002, 'learning_rate': 2.054794520547945e-06, 'epoch': 4.79}\n",
      "{'loss': 0.0184, 'learning_rate': 1.598173515981735e-06, 'epoch': 4.84}\n",
      "{'loss': 0.0053, 'learning_rate': 1.1415525114155251e-06, 'epoch': 4.89}\n",
      "{'loss': 0.0302, 'learning_rate': 6.849315068493151e-07, 'epoch': 4.93}\n",
      "{'loss': 0.0094, 'learning_rate': 2.2831050228310502e-07, 'epoch': 4.98}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18f836fca723436e859cf3bafddcc179",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8587970733642578, 'eval_accuracy': 0.8996138996138996, 'eval_f1': 0.9277777777777778, 'eval_runtime': 3.9267, 'eval_samples_per_second': 197.876, 'eval_steps_per_second': 3.311, 'epoch': 5.0}\n",
      "{'train_runtime': 307.0645, 'train_samples_per_second': 113.787, 'train_steps_per_second': 3.566, 'train_loss': 0.015705070360914536, 'epoch': 5.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1095, training_loss=0.015705070360914536, metrics={'train_runtime': 307.0645, 'train_samples_per_second': 113.787, 'train_steps_per_second': 3.566, 'train_loss': 0.015705070360914536, 'epoch': 5.0})"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a197c923",
   "metadata": {},
   "source": [
    "**模型评估**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9784bec1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c70d869a3a9f4ce9bd3565e16d307208",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/110 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.021800853312015533,\n",
       " 'eval_accuracy': 0.9935603892386949,\n",
       " 'eval_f1': 0.9952914094381082,\n",
       " 'eval_runtime': 27.1377,\n",
       " 'eval_samples_per_second': 257.502,\n",
       " 'eval_steps_per_second': 4.053,\n",
       " 'epoch': 5.0}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"评估训练集\"\"\"\n",
    "trainer.evaluate(tokenized_datasets[\"train\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5f1a2d5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "680eec767f024afd8281133e68758afc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.6429142951965332,\n",
       " 'eval_accuracy': 0.9099099099099099,\n",
       " 'eval_f1': 0.9351851851851852,\n",
       " 'eval_runtime': 4.1169,\n",
       " 'eval_samples_per_second': 188.732,\n",
       " 'eval_steps_per_second': 3.158,\n",
       " 'epoch': 5.0}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"评估测试集\"\"\"\n",
    "trainer.evaluate(tokenized_datasets[\"test\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eea30eb1",
   "metadata": {},
   "source": [
    "**模型预测**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b1998040",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3827e400a086440fb2f82370b7179028",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "PredictionOutput(predictions=array([[-5.2204757,  5.654408 ],\n",
       "       [-5.414548 ,  5.956816 ],\n",
       "       [-5.172721 ,  5.7040668],\n",
       "       ...,\n",
       "       [-5.5483527,  6.0281124],\n",
       "       [-3.4442828,  3.8156376],\n",
       "       [ 5.2646294, -5.8420653]], dtype=float32), label_ids=array([1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,\n",
       "       1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1,\n",
       "       0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1,\n",
       "       1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1,\n",
       "       1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1,\n",
       "       0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,\n",
       "       1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1,\n",
       "       0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1,\n",
       "       1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1,\n",
       "       1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1,\n",
       "       1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0,\n",
       "       1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1,\n",
       "       1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1,\n",
       "       1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1,\n",
       "       0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1,\n",
       "       1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1,\n",
       "       0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1,\n",
       "       1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1,\n",
       "       0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1,\n",
       "       1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1,\n",
       "       1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1,\n",
       "       1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0,\n",
       "       1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1,\n",
       "       1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1,\n",
       "       0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0,\n",
       "       0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0,\n",
       "       0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1,\n",
       "       1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0,\n",
       "       1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1,\n",
       "       1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0,\n",
       "       0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0,\n",
       "       1, 1, 0, 0, 1, 1, 0], dtype=int64), metrics={'test_loss': 0.6429142951965332, 'test_accuracy': 0.9099099099099099, 'test_f1': 0.9351851851851852, 'test_runtime': 4.0297, 'test_samples_per_second': 192.817, 'test_steps_per_second': 3.226})"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.predict(tokenized_datasets[\"test\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d08f10c6",
   "metadata": {},
   "source": [
    "**可视化**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f4f6d2e",
   "metadata": {},
   "source": [
    "进入checkpoints文件夹路径，激活虚拟环境，输入：tensorboard --logdir runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88c45539",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.21"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
